# -*- coding: utf-8 -*-
import urllib2
#import cookielib
from urllib import urlencode
import zipfile
import mechanize
import logging
from random import choice
import csv
import os, sys
import traceback
import time
import threading
import Queue
from datetime import datetime, timedelta
import socket
from anatel_utils import random_proxy_italo,random_header
from proxy51 import get_all_proxy51

timeout = 60
socket.setdefaulttimeout(timeout)

date_now = datetime.now()
dir_name = "" + date_now.strftime("%Y%m%d%H%M%S")
cust_dir_name=0

estado_array = ['AC','AL','AP','AM','BA','CE','DF','ES','GO','MA','MT','MS','MG','PA','PB','PR','PE','PI','RJ','RN','RS','RO','RR','SC','SP','SE','TO']    
freq_array = ['414:448','1427:1517','3800:4200','4400:5000','5925:6425','6430:7110','7425:7725','7725:8275','8275:8500','10700:11700','12200:13250','14500:15350','17700:19700','21300:23600','37000:39500']
div_const=2000
workers_param = 50

use_proxy = 1 #Usa Proxy por Padrão

"""
proxy_ips = [
'187.58.200.182:80',
'187.5.159.218:80',
'187.33.255.241:3128',
'187.66.149.165:3128',
'187.133.89.101:3128',
'187.21.135.3:3128',
'187.33.88.117:3128',
'187.32.127.163:3128',
'187.108.85.129:3128',
'187.61.213.3:3128',
'187.59.5.190:3128',
'187.115.161.139:3128',
'187.141.248.178:3128',
'187.84.208.58:3128',
'187.67.29.51:3128',
'187.16.140.137:3128',
'187.109.10.73:3128',
'187.125.38.20:3128',
'187.123.82.227:3128',
'187.28.167.180:3128',
            ]

proxy_fail_count = {
'187.58.200.182:80':0,
'187.5.159.218:80':0,
'187.33.255.241:3128':0,
'187.66.149.165:3128':0,
'187.133.89.101:3128':0,
'187.21.135.3:3128':0,
'187.33.88.117:3128':0,
'187.32.127.163:3128':0,
'187.108.85.129:3128':0,
'187.61.213.3:3128':0,
'187.59.5.190:3128':0,
'187.115.161.139:3128':0,
'187.141.248.178:3128':0,
'187.84.208.58:3128':0,
'187.67.29.51:3128':0,
'187.16.140.137:3128':0,
'187.109.10.73:3128':0,
'187.125.38.20:3128':0,
'187.123.82.227:3128':0,
'187.28.167.180:3128':0,
            }
"""

proxy_ips = get_all_proxy51('http://www.proxy51.com/api/f9448939cd187cbd7f921f1642e2d826')

pool_free_proxy_ips = proxy_ips
pool_used_proxy_ips = []

proxy_fail_count = {}
for proxy_ip in proxy_ips:
    proxy_fail_count[str(proxy_ip)] = 0
    
def rm_proxy(proxy_to_be_deleted):
    for rm_proxy in range(len(pool_free_proxy_ips)):
        if pool_free_proxy_ips[rm_proxy]==proxy_to_be_deleted:
            print "****DELETADO Proxy %s ***" % (pool_free_proxy_ips[rm_proxy])
            #del pool_free_proxy_ips[rm_proxy]
            #return pool_free_proxy_ips
            break
        
def free_proxy(proxy_to_free):
    for rm_proxy in range(len(pool_used_proxy_ips)):
        if pool_used_proxy_ips[rm_proxy]==proxy_to_free:
            print "****LIBERADO Proxy %s ***" % (pool_used_proxy_ips[rm_proxy])            
            del pool_used_proxy_ips[rm_proxy]
            pool_free_proxy_ips.append(proxy_to_free)
            print "****Proxies Livres na Pool: %s***" % (len(pool_free_proxy_ips))
            print "****Proxies Usados na Pool: %s***" % (len(pool_used_proxy_ips))
            #return pool_free_proxy_ips,pool_used_proxy_ips
            break        
        
def reserve_proxy(proxy_to_reserve):
    for rm_proxy in range(len(pool_free_proxy_ips)):
        if pool_free_proxy_ips[rm_proxy]==proxy_to_reserve:
            print "****RESERVADO Proxy %s ***" % (pool_free_proxy_ips[rm_proxy])
            del pool_free_proxy_ips[rm_proxy]
            pool_used_proxy_ips.append(proxy_to_reserve)
            print "****Proxies Livres na Pool: %s***" % (len(pool_free_proxy_ips))
            print "****Proxies Usados na Pool: %s***" % (len(pool_used_proxy_ips))            
            #return pool_free_proxy_ips,pool_used_proxy_ips
            break       
    

def genAnatelURL(uf,fr,DIVISAO_CONST):
            url_array = []
            filename_array = []
            
            fr_inicial = fr.split(':')[0]
            fr_final = fr.split(':')[1]
            
            
            range_int = ((int(fr_final)-int(fr_inicial))/DIVISAO_CONST)+1
            #print "RangeInt: %s" % (range_int)
            for i in range(0,range_int):
                #print "i = %s" % (i)
                fr_inicial_loop = int(fr_inicial)+(i*DIVISAO_CONST)
                if int(fr_final) < int(fr_inicial)+((i+1)*DIVISAO_CONST):
                    fr_final_loop = int(fr_final)
                else:
                    fr_final_loop = int(fr_inicial)+((i+1)*DIVISAO_CONST)
                    
                if fr_inicial_loop==fr_final_loop:
                    break
                           
                #print "Inicial:%s  Final:%s" % (fr_inicial_loop,fr_final_loop)
                #template_url='http://sistemas.anatel.gov.br/stel/Consultas/RecuperacaoFrequencias/tela.asp?SISQSmodulo=9896&IndFiltro=u&IndApresentacao=i&pMedTransmissaoInicial=%s&pMedRecepcaoInicial=%s&pIdtUnidadeTransmissao=2&pIdtUnidadeRecepcao=2&SiglaUF=%s&IndGrupoEstacao=r&botaoFlatDownloadTXT="true"&acao=t' % (fr_inicial_loop,fr_final_loop,uf)
                template_url='http://sistemas.anatel.gov.br/stel/Consultas/RecuperacaoFrequencias/tela.asp'
                #print template_url
                '''
                txdata = urlencode({
                            'SISQSmodulo':'9896',
                            'IndFiltro':'u',
                            'IndApresentacao':'i',
                            'pMedTransmissaoInicial':fr_inicial_loop,
                            'pMedRecepcaoInicial':fr_final_loop,
                            'pIdtUnidadeTransmissao':'2',
                            'pIdtUnidadeRecepcao':'2',
                            'SiglaUF': uf,
                            'IndGrupoEstacao':'r',
                            'botaoFlatDownloadTXT':'"true"',
                            'acao':'t',
                   })
                '''   
                vars_html = {'SISQSmodulo':'9896',
                            'IndFiltro':'u',
                            'IndApresentacao':'i',
                            'pMedTransmissaoInicial':fr_inicial_loop,
                            'pMedRecepcaoInicial':fr_final_loop,
                            'pIdtUnidadeTransmissao':'2',
                            'pIdtUnidadeRecepcao':'2',
                            'SiglaUF': uf,
                            'IndGrupoEstacao':'r',
                            'botaoFlatDownloadTXT':'true',
                            'acao':'t',}
                
                
                filename = uf+'_'+str(fr_inicial_loop)+'_'+str(fr_final_loop)+'.zip'
                url=template_url+'?'
                
                for key,value in vars_html.items():
                    url = url + str(key) + '=' + str(value) + '&'
                
                
                url_array.append(url[:-1])
                filename_array.append(filename)
                
                
            return url_array,filename_array

def mkDirLinux(dir_name):
    print execServerLit("mkdir %s" % (dir_name))

def execServerLit(command):
    execute_command = os.popen(command,"r")
    return execute_command.read()

def dlfile(url,local_filename):
    
    # Open the url
    br = mechanize.Browser()
    br.set_handle_equiv(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(False)
    br.set_handle_gzip(False)
    br.set_handle_robots(False)
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=timeout)    
    try:
        '''
        txheaders = {'User-agent' : random_header()}
        cj = cookielib.CookieJar()
        proxy_random = random_proxy_italo()
        proxy_handler = ProxyHandler({'http': proxy_random})
        opener = build_opener(HTTPCookieProcessor(cj),proxy_handler)
        install_opener(opener)
        req = Request(url, txdata, txheaders)
        f = urlopen(req,timeout=timeout)
        '''
    
        br.addheaders = [('User-agent', random_header())]
        proxy_random = ""
        if use_proxy:
            #proxy_random = random_proxy_italo()            
            proxy_random = choice(pool_free_proxy_ips)
            reserve_proxy(proxy_random)
            
            #check if proxy has password in the string or not
            if proxy_random.strip().split(":")>2:
                proxy_random_connect = proxy_random.strip().split(":")[-2]+":"+proxy_random.strip().split(":")[-1]+"@"+proxy_random.strip().split(":")[0]+":"+proxy_random.strip().split(":")[1]
            else:
                proxy_random_connect = proxy_random
                
            br.set_proxies({"http": proxy_random_connect})   
            #print "proxy: %s" % (proxy_random_connect)
        
        #print "downloading " + url
        cj = mechanize.CookieJar()
        br.set_cookiejar(cj) 
                    
        #br.set_debug_redirects(True)
        #br.set_debug_responses(True)
        #br.set_debug_http(True)
        
        # To make sure you're seeing all debug output:
        #logger = logging.getLogger("mechanize")
        #logger.addHandler(logging.StreamHandler(sys.stdout))
        #logger.setLevel(logging.DEBUG) 

        #req = urllib2.Request(url, txdata, {'User-agent': random_header()})
        #br.open(req)

        br.retrieve(url,(dir_name + "/" + local_filename))[0]

        '''
        # Open our local file for writing
        with open(os.path.basename(dir_name + "/" + local_filename), "wb") as local_file:
            local_file.write(f.read())
        '''
        tamanho=os.path.getsize(dir_name + "/" + local_filename)
        print ("Tamanho(%s): %s" % (local_filename,tamanho))
        
        
        if tamanho == 376:
            print "Excedeu o tamanho (%s apagado!)" % (local_filename)
            #apaga o arquivo
            os.remove(dir_name + "/" + local_filename)
            #splitando 
            uf_l = local_filename.split("_")[0]
            DIV_l = int(local_filename.split("_")[2][:-4])-int(local_filename.split("_")[1])
            fr_l = local_filename.split("_")[1] + ":" + local_filename.split("_")[2][:-4]
            print "uf_l: %s" % (uf_l)
            print "DIV_l: %s" % (DIV_l)
            print "fr_l: %s" % (fr_l)
            
            url_arr,filename_arr = genAnatelURL(uf_l,fr_l,(DIV_l/2)) #divide o intervalo por dois
            for url_i in range(len(url_arr)):
                #queue.put([url_arr[url_i],filename_arr[url_i]])
                #dlfile(url_arr[url_i],filename_arr[url_i])
                dlfile_manager(url_arr[url_i],filename_arr[url_i])                
                print "filename_arr: %s" % (filename_arr[url_i]) 

            
        elif tamanho < 400:
            #dlfile(url,local_filename)
            return False
        elif tamanho == 619:
            print "ARQUIVO VAZIO"
            return True
        else:
            file_id = open(date_now.strftime("%Y-%m-%d")+"_gen_files.txt", "a")
            file_id.write(local_filename+"\n")
            file_id.close()
            
            #TESTA O ARQUIVO ZIP, caso raise um erro, ele vai pro except e dá download de novo
            fileZiptest = zipfile.ZipFile(dir_name + "/" + local_filename, "r")
            fileZiptest.testzip()
            fileZiptest.close()
            return True
            
        
    #handle errors
    except mechanize.HTTPError:
        #exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
        #print "%s:%s" % (exceptionType,exceptionValue)            
        if use_proxy:
            free_proxy(proxy_random)
            proxy_fail_count[proxy_random]=proxy_fail_count[proxy_random]+1
            for k,v in proxy_fail_count.items():
                if v >= 5:
                    rm_proxy(k)
                    
        #dlfile(url,local_filename)
        return False
           
    except:        
        #print "HTTP Error:", e.code, url
        exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
        #traceback.print_exception(exceptionType, exceptionValue, exceptionTraceback,
        #                  limit=2, file=sys.stdout)
        #logger.error(str(exceptionValue))
        print "%s:%s" % (exceptionType,exceptionValue)
        if use_proxy:
            free_proxy(proxy_random)   
        #dlfile(url,local_filename)
        return False
    finally:
        if use_proxy:
            free_proxy(proxy_random)
        br.close()
    
    return True #se passar por tudo retorna verdadeiro (sucesso)        
        
def dlfile_manager(url,local_filename):
    try:
        bool_success=False
        while not (bool_success):
            bool_success = dlfile(url,local_filename)
    except:
        exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
        print "%s:%s" % (exceptionType,exceptionValue)
        sys.exit(1)        
        
            
if not len(sys.argv) < 2:
    
    for v in sys.argv:
        if len(v.split("="))>1:
            param = v.split("=")[0].strip()
            valor = v.split("=")[1].strip()
            try:
                if param.lower().strip().find("estados")>-1:
                    estados = valor
                    #print "url_play: %s" % (url_play) 
                    if estados.find(",")>-1:
                        estado_array = estados.split(",")
                        for estado_arr in estado_array:
                            print "estado: %s" % (estado_arr)
                    else:
                        estado_array = [estados]
                        print "estado: %s" % (estado_array)
                elif param.lower().strip().find("freq")>-1:
                    freq = valor
                    #print "url_play: %s" % (url_play) 
                    if freq.find(",")>-1:
                        freq_array = freq.split(",")
                        for freq_arr in freq_array:
                            print "freq: %s" % (freq_arr)
                    else:
                        freq_array = [freq]
                        print "freq: %s" % (freq_array)
                elif param.lower().strip().find("div_const")>-1:
                    div_const = valor
                    print "div_const: %s" % (div_const)
                elif param.lower().strip().find("workers")>-1:
                    workers_param = int(valor)
                    print "workers: %s" % (workers_param)                           
                elif param.lower().strip().find("use_proxy")>-1:
                    use_proxy = int(valor)
                    print "use_proxy: %s" % (use_proxy)                    
                elif param.lower().strip().find("dir_name")>-1:
                    dir_name = valor
                    cust_dir_name = 1
                    print "dir_name: %s" % (dir_name)                                                                       
                else:
                    eval('%s=%s' % (param,valor))   
                #print "Param: %s" % (param)
                #print "Valor: %s" % (valor)
            except:
                exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
                traceback.print_exception(exceptionType, exceptionValue, exceptionTraceback,
                                  limit=2, file=sys.stdout)
                sys.exit(1)  
   
try:      

    WORKERS = workers_param
    
    class Worker(threading.Thread):
    
        def __init__(self, queue):
            self.__queue = queue
            threading.Thread.__init__(self)
    
        def run(self):
            while 1:
                item = self.__queue.get()
                if item[0] is None:
                    break # reached end of queue
                
                url = item[0]
                filename = item[1]
                
                try:
                    start = time.time()  
                    
                    dlfile_manager(url,filename)
                    
                except IOError, e:
                    print "Nao foi possivel baixar o zip!"
                    print e
                    sys.exit(1)
                except zipfile.error, e:
                    print "Erro ao descompactar, nao eh zip!"
                    print e
                    sys.exit(1)
                
                task_message = "task ", item[1], " finished! (Tempo Gasto: %s)" % ((time.time() - start))
                print task_message
    
    #caso o dir_name for colocado no inicio nao criar diretorio
    if not cust_dir_name:
        mkDirLinux(dir_name)

    queue = Queue.Queue(0)
    

    for i in range(WORKERS):
        Worker(queue).start() # start a worker
    
    '''
    ifile  = open('download.csv', "rb")
    reader = csv.reader(ifile)

    for row in reader:
        filename = row[0]+'_'+row[1]+'_'+row[2]+'-'+row[3]+'.zip'
        url=row[-1]
        queue.put([url,filename])
    '''
    
    #'''

    for uf in estado_array:
    #for uf in ['AC']: #!!!!!!!**********TEST ONLY********!!!!!!!!!!!!
        for fr in freq_array:
        #for fr in ['414:448']:#!!!!!!!**********TEST ONLY********!!!!!!!!!!!!
            url_arr,filename_arr = genAnatelURL(uf,fr,div_const)
            for url_i in range(len(url_arr)):
                queue.put([url_arr[url_i],filename_arr[url_i]])
                
    #'''
    
    for i in range(WORKERS):
        queue.put([None,None]) # add end-of-queue markers

    #ifile.close()
    
except KeyboardInterrupt:
    print('Interrompendo!!')
    sys.exit(1)
except:
    exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
    traceback.print_exception(exceptionType, exceptionValue, exceptionTraceback,
                      limit=2, file=sys.stdout)
    print "%s:%s" % (exceptionType,exceptionValue)
    sys.exit(1)